#!/bin/bash
# Copyright (c) Los Alamos National Security, LLC, and others.

# Make sure the Twitter preprocessing works.

. ./environment.sh

cd $DATADIR
exec 2>&1

parsesh=$QUACBASE/misc/parse.sh

mkdir raw pre geo raw/foo
ln -s $TWEETDIR/raw/small/small.json.gz raw/foo
touch raw/foo/small.stats
touch -d19700101 raw/foo/small.json.d  # suppress "No such file or directory"

# lots of order variability in this command, so ignore the output
z $parsesh 1 . > /dev/null

x ls -R .

y 'wc -l pre/*.tsv'
y 'md5sum pre/*.tsv'
x pickle-dump pre/metadata
